{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd \n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> If havs Cuda</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "class ReplayBuffer(object):\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = deque(maxlen=capacity)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        state      = np.expand_dims(state, 0)\n",
    "        next_state = np.expand_dims(next_state, 0)\n",
    "            \n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        state, action, reward, next_state, done = zip(*random.sample(self.buffer, batch_size))\n",
    "        return np.concatenate(state), action, reward, np.concatenate(next_state), done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Computing Temporal Difference Loss</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_td_loss(batch_size):\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    state      = Variable(torch.FloatTensor(np.float32(state)))\n",
    "    next_state = Variable(torch.FloatTensor(np.float32(next_state)), volatile=True)\n",
    "    action     = Variable(torch.LongTensor(action))\n",
    "    reward     = Variable(torch.FloatTensor(reward))\n",
    "    done       = Variable(torch.FloatTensor(done))\n",
    "\n",
    "    q_values      = model(state)\n",
    "    next_q_values = model(next_state)\n",
    "\n",
    "    q_value          = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n",
    "    next_q_value     = next_q_values.max(1)[0]\n",
    "    expected_q_value = reward + gamma * next_q_value * (1 - done)\n",
    "    \n",
    "    loss = (q_value - Variable(expected_q_value.data)).pow(2).mean()\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_game(frame_idx, num_frames, rewards, losses, img):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s / %s. reward: %s' % (frame_idx, num_frames, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    \n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    \n",
    "    plt.subplot(133)\n",
    "    plt.title('img')\n",
    "    plt.imshow(img)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Atari Environment</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from common.wrappers import make_atari, wrap_deepmind, wrap_pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env)\n",
    "env    = wrap_pytorch(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CnnDQN(nn.Module):\n",
    "    def __init__(self, input_shape, num_actions):\n",
    "        super(CnnDQN, self).__init__()\n",
    "        \n",
    "        self.input_shape = input_shape\n",
    "        self.num_actions = num_actions\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(self.feature_size(), 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, self.num_actions)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "    \n",
    "    def feature_size(self):\n",
    "        return self.features(autograd.Variable(torch.zeros(1, *self.input_shape))).view(1, -1).size(1)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            state   = Variable(torch.FloatTensor(np.float32(state)).unsqueeze(0), volatile=True)\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0]\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CnnDQN(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(model.parameters(), lr=0.00001)\n",
    "\n",
    "replay_initial = 10000\n",
    "replay_buffer = ReplayBuffer(100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 30000\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x13b3fa5d0>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAYT0lEQVR4nO3de3AV53nH8e9zjnR0F0JIGIFkAy6JjV3bsTU2bnpxm4uxm8BM27QwSZO0aTy9uLdk2rEnrdu4/zRNJ9OmIY3dNEmbiePYSSelCRmmjem0TWMHuTYOYHAExkYYjAwCAQJ0e/rHrvCxOJcVHGm1e36fGY12391z9llW/PTq3T275u6IiEh6ZeIuQEREZpeCXkQk5RT0IiIpp6AXEUk5Bb2ISMrVxLXhjo4OX758eVybFxFJpKeffvo1d++cyWtiC/rly5fT19cX1+ZFRBLJzF6a6Ws0dCMiknIKehGRlFPQi4iknIJeRCTlFPQiIilXNujN7AtmdtTMdhZZbmb2aTPrN7PnzOzmypcpIiKXKkqP/kvA2hLL7wJWhV/3AH9/+WWJiEillA16d/8v4HiJVdYD/+yBJ4E2M+uqVIHTbT9wnE9u3cPEpG6vLCISRSXG6JcBB/PmB8K2i5jZPWbWZ2Z9g4ODl7SxZ18+waZt+xgZHb+k14uIVJs5PRnr7g+7e6+793Z2zugTvBc01QUf5j1zfqKSpYmIpFYlgv4Q0JM33x22zYqmuiwAp8+rRy8iEkUlgn4z8P7w6ps1wEl3P1yB9y2o+UKPXkEvIhJF2ZuamdlXgTuADjMbAP4MqAVw988BW4C7gX5gBPi12SoW8oZuNEYvIhJJ2aB3941lljvwOxWrqIymnMboRURmInGfjJ0ao9fQjYhINIkL+qkxep2MFRGJJnFB36STsSIiM5K4oG+ozWIGZ0Y1Ri8iEkXigj6TMRprs+rRi4hElLigh2D4RkEvIhJNIoO+ua5GJ2NFRCJKZNCrRy8iEl1Cgz6rk7EiIhElM+hz6tGLiESVzKDX0I2ISGSJDfrTuteNiEgkiQz65rqsnjAlIhJRIoO+qa6GkdEJJvXcWBGRspIZ9Dndk15EJKpkBr2eGysiEllCg17PjRURiSqRQT91T3qdkBURKS+RQd+Y08NHRESiSmTQN2uMXkQkskQGvZ4bKyISXSKDXs+NFRGJLpFB36STsSIikSUy6KeeG3v6nIJeRKScRAZ9JmM052o4paEbEZGyEhn0AC31NZxSj15EpKwEB30tp86NxV2GiMi8l+CgV49eRCSKRAf9sHr0IiJlJTjoa9WjFxGJIMFBr6EbEZEoIgW9ma01s71m1m9m9xVYfqWZbTOzZ8zsOTO7u/KlvtHUyVh3PWVKRKSUskFvZllgE3AXsBrYaGarp632J8Bj7v4WYAPw2UoXOl1LfQ1jE8758cnZ3pSISKJF6dHfCvS7+353HwUeBdZPW8eB1nB6AfBK5UosrLU+uA2CTsiKiJQWJeiXAQfz5gfCtnx/DrzPzAaALcDvFnojM7vHzPrMrG9wcPASyn1da0MtgMbpRUTKqNTJ2I3Al9y9G7gb+LKZXfTe7v6wu/e6e29nZ+dlbbAl7NEr6EVESosS9IeAnrz57rAt34eAxwDc/ftAPdBRiQKLaamf6tFr6EZEpJQoQb8dWGVmK8wsR3CydfO0dV4G3gZgZtcSBP3ljc2UoR69iEg0ZYPe3ceBe4GtwPMEV9fsMrMHzWxduNpHgQ+b2Q7gq8AHfZave1SPXkQkmpooK7n7FoKTrPltD+RN7wbeWtnSSlOPXkQkmsR+MrY5V4MZDCvoRURKSmzQX3j4iIZuRERKSmzQg+53IyISRcKDXg8fEREpJ+FBrx69iEg5CnoRkZRLeNBr6EZEpJyEB7169CIi5SQ86GsZ1sNHRERKSnTQtzXWMjbhnB2biLsUEZF5K9lBH96T/sSIxulFRIpJdNAvCIP+5FkFvYhIMckO+kb16EVEykl20F/o0Y/GXImIyPyV6KBva8wBGroRESkl2UGvk7EiImUlOugbc1lqMqYevYhICYkOejOjrbGWEwp6EZGiEh30EJyQPamhGxGRotIR9OrRi4gUlfigb2vMcUKXV4qIFJX8oG+o1VU3IiIlJD7oWzV0IyJSUuKDvq2xllPnxhmfmIy7FBGReSnxQT91G4RhPYBERKSgxAd9W6PuYCkiUkryg74huN/NiRFdeSMiUkjig7516n436tGLiBSU+KCfGroZVtCLiBSU/KAPe/RDZzR0IyJSSOKDfuqqmyF9aEpEpKBIQW9ma81sr5n1m9l9Rdb5ZTPbbWa7zOyRypZZXE02Q1tjLcfVoxcRKaim3ApmlgU2Ae8ABoDtZrbZ3XfnrbMKuB94q7sPmdni2Sq4kPamnIJeRKSIKD36W4F+d9/v7qPAo8D6aet8GNjk7kMA7n60smWW1t6ooBcRKSZK0C8DDubND4Rt+d4EvMnMvmdmT5rZ2kJvZGb3mFmfmfUNDg5eWsUFqEcvIlJcpU7G1gCrgDuAjcA/mFnb9JXc/WF373X33s7OzgptGhY15zimoBcRKShK0B8CevLmu8O2fAPAZncfc/cXgRcIgn9OtDflGBoZxd3napMiIokRJei3A6vMbIWZ5YANwOZp63yToDePmXUQDOXsr2CdJS1szDEx6Qyf1Y3NRESmKxv07j4O3AtsBZ4HHnP3XWb2oJmtC1fbChwzs93ANuCP3P3YbBU93aLm4H43x86cn6tNiogkRtnLKwHcfQuwZVrbA3nTDnwk/Jpz7U11AAzpxmYiIhdJ/CdjIbi8EuDYaQW9iMh06Qj6cOhGl1iKiFwsHUEf9uiPa+hGROQiqQj6hlyWhtosxzV0IyJykVQEPejTsSIixaQm6Bc15zR0IyJSQGqCfqFubCYiUlBqgn5RU06XV4qIFJCaoO9oqeO10+d1vxsRkWlSE/SLW+o4Pz7JqfO6342ISL7UBH1nS3AbhMFTut+NiEi+9AR9cxD0R4cV9CIi+dIT9FM9+tMKehGRfKkJ+sUt9YCGbkREpktN0Lc21JDLZjh66lzcpYiIzCupCXozo7OlTj16EZFpUhP0EFxLr6AXEXmjVAX9YgW9iMhFUhX0GroREblYuoK+uY7jI6OMTUzGXYqIyLyRqqBf3FqHu54dKyKSL1VBP/XpWA3fiIi8Ll1Bf+HTsbqWXkRkSqqCfnFr8OnYV3W/GxGRC9IV9C11ZAwOn1SPXkRkSqqCvjabobOljsMnzsZdiojIvJGqoAfoWtCgHr2ISJ4UBn09h0+qRy8iMiWFQR/06PXsWBGRQAqDvp6R0QmGz+nZsSIikMagbwsusdTwjYhIIFLQm9laM9trZv1mdl+J9X7RzNzMeitX4sx0LZgKep2QFRGBCEFvZllgE3AXsBrYaGarC6zXAvw+8FSli5yJrgUNABw+oaAXEYFoPfpbgX533+/uo8CjwPoC6/0F8Akg1oSd+tDUEQ3diIgA0YJ+GXAwb34gbLvAzG4Getz926XeyMzuMbM+M+sbHByccbFR1GQzLG6p5xUN3YiIABU4GWtmGeBTwEfLrevuD7t7r7v3dnZ2Xu6mi+pqq+eIgl5EBIgW9IeAnrz57rBtSgtwPfCfZnYAWANsjvOE7NIFDbyi2yCIiADRgn47sMrMVphZDtgAbJ5a6O4n3b3D3Ze7+3LgSWCdu/fNSsURdLc3MDB0lslJfWhKRKRs0Lv7OHAvsBV4HnjM3XeZ2YNmtm62C7wUPQsbGZ2Y5NVTGr4REamJspK7bwG2TGt7oMi6d1x+WZfnyvZGAA4eP3vhcksRkWqVuk/GAvSEQf/y8ZGYKxERiV8qg35ZWwNmCnoREUhp0OdqMnS11jOgoBcRSWfQQzB8ox69iEjKg/7gkIJeRCS1QX9leyOvDp/n3NhE3KWIiMQqtUHf0x5cVjkwpE/Iikh1S23QX3nhEsszMVciIhKv1Ab98kVNAOwfVNCLSHVLbdC3N+Voa6xln4JeRKpcaoPezFjZ0cT+wdNxlyIiEqvUBj3A1Z3N7H9NPXoRqW6pDvqVnc0MnjrP8LmxuEsREYlNyoNeJ2RFRFId9Fd3NgNonF5Eqlqqg/7K9kayGVOPXkSqWqqDPleT4cr2RvapRy8iVSzVQQ/B8M0Lr56KuwwRkdikPuiv7WrhxdfO6OZmIlK1qiDoW5l06D+q4RsRqU6pD/prlrQAsPvwcMyViIjEI/VBf9WiJuprM+w5rHF6EalOqQ/6bMZ485JW9hxRj15EqlPqgx7g2iUtPH94GHePuxQRkTlXFUF/zZIWhkbGOHrqfNyliIjMuaoI+mu7WgHY9crJmCsREZl7VRH01y9bQMbg2YMKehGpPlUR9E11NbzpihZ2HDwRdykiInOuKoIe4KaeNnYMnNAJWRGpOlUT9Df2tHFiZIyXjo3EXYqIyJyqmqC/qacNgB0DGr4RkeoSKejNbK2Z7TWzfjO7r8Dyj5jZbjN7zsy+a2ZXVb7Uy7NqcTMNtVmeeVlBLyLVpWzQm1kW2ATcBawGNprZ6mmrPQP0uvsNwNeBv6p0oZerJpvhhu4FPP3SUNyliIjMqSg9+luBfnff7+6jwKPA+vwV3H2bu08Nfj8JdFe2zMq4beUidr1yUg8LF5GqEiXolwEH8+YHwrZiPgR8p9ACM7vHzPrMrG9wcDB6lRWyZmU7kw59B47P+bZFROJS0ZOxZvY+oBf4ZKHl7v6wu/e6e29nZ2clNx3JzVcuJJfN8OR+Bb2IVI+aCOscAnry5rvDtjcws7cDHwN+xt3n5U1l6muz3NTTxpP7j8VdiojInInSo98OrDKzFWaWAzYAm/NXMLO3AA8B69z9aOXLrJw1K9vZeUjj9CJSPcoGvbuPA/cCW4HngcfcfZeZPWhm68LVPgk0A4+b2bNmtrnI28Xu9qs7mHT4/j716kWkOkQZusHdtwBbprU9kDf99grXNWt6ly+kpa6GbXuOcud1S+IuR0Rk1lXNJ2On1GYz/NSbOti296jueyMiVaHqgh7g5665gleHz7PrFT1eUETSryqD/o43d2IG2/bM6/PGIiIVUZVB39Fcx009bWzdfSTuUkREZl1VBj3Az/94FzsPDbN/8HTcpYiIzKqqDfp33bAUM/i3HYfjLkVEZFZVbdAvWVDPrcvb2bzjkK6+EZFUq9qgB1h301L2DZ5h5yFdfSMi6VXVQf+uG5ZSX5vhkR+8FHcpIiKzpqqDfkFDLetuXMq/PvuK7n0jIqlV1UEP8L41VzEyOsE3n7nohpwiIqlQ9UF/Q3cbN3Qv4IvfO8DEpE7Kikj6VH3QA/zWz1zNi6+d4ds/1KWWIpI+CnrgzuuW8GOLm9n0RD+T6tWLSMoo6IFMxvidn72ava+eUq9eRFJHQR9ad+Myru1q5S+/s4dzYxNxlyMiUjEK+lA2Y/zpu67l0ImzfP6/98ddjohIxSjo8/zE1R3cdf0S/u6JfvqP6mZnIpIOCvppPr7+OhpyWT76+A7GJybjLkdE5LIp6KdZ3FLPX6y/nh0HT/Cpf38h7nJERC6bgr6Ad9+4lI239vDZ/9zHt5/TVTgikmwK+iI+vu56brlqIR99/Fme3H8s7nJERC6Zgr6IXE2Gh371FroXNvLrX9pO34HjcZckInJJFPQldDTX8chv3MYVrfW89/NPsUUfphKRBFLQl7G4tZ7Hf/N2rlvaym9/5f/46617GdPVOCKSIAr6CDqa63jkw2t4zy3dfGZbP7/w2f9l56GTcZclIhKJgj6i+tosn3zPjXzufTdz6MRZ3v2Z/+GPHt/BweMjcZcmIlJSTdwFJM3a67u4/eoOPrutny9+7wDf+L8B7rxuCe+97Spuv3oR2YzFXaKIyBuYezy35e3t7fW+vr5Ytl0pR06e45++f4CvPPkSw+fG6Wiu467rl/BTqzq4beUiFjTUxl2iiKSMmT3t7r0zeo2C/vKdG5vgiT1H+bcdr7Bt71HOjU2SMbhmSSvXLW1l9dJWrlnSypWLGlnSWq9ev4hcMgX9PHB+fIJnXz7B9/Yd45mXh3j+8DCvnR69sLwmYyxta6BrQT2LmnMsbMyxqCnHwqYcCxpqacxlacjV0FCbDaeD77XZDDUZo2bqe8bIZgwz/dIQqSaXEvSRxujNbC3wt0AW+Ly7/+W05XXAPwO3AMeAX3H3AzMpJC3qarLctnIRt61cdKHt6KlzvHDkNAeHRjh4fISBobMcPnmWvUdOMTQyxtDIKJf6+3Yq8GuzmTD4wQAze/37hTYw3rgOU+0GmfzXQLDSPDBPypgXv1Tjr0Aq4ffetop337h0zrZXNujNLAtsAt4BDADbzWyzu+/OW+1DwJC7/5iZbQA+AfzKbBScRItb6lncUl90+cSkM3x2jJNnxzg7NsHI6ARnRycYGR3n7FgwPTYxyfikMz7h4fdJxiadicnJN7Q54A6Oh9/BPZy+qD2YJ2+9ybzp+WB+VMG8KMTnQxFSEXN9/i5Kj/5WoN/d9wOY2aPAeiA/6NcDfx5Ofx34jJmZz5e0mOeyGWNhOHwjIlJpUa6jXwYczJsfCNsKruPu48BJYNG0dTCze8ysz8z6BgcHL61iERGZkTn9wJS7P+zuve7e29nZOZebFhGpWlGC/hDQkzffHbYVXMfMaoAFBCdlRUQkZlGCfjuwysxWmFkO2ABsnrbOZuAD4fQvAU9ofF5EZH4oezLW3cfN7F5gK8HllV9w911m9iDQ5+6bgX8Evmxm/cBxgl8GIiIyD0S6jt7dtwBbprU9kDd9DnhPZUsTEZFK0N0rRURSTkEvIpJysd3rxswGgZcu8eUdwGsVLCcJtM/VQftcHS5nn69y9xldnx5b0F8OM+ub6U19kk77XB20z9VhrvdZQzciIimnoBcRSbmkBv3DcRcQA+1zddA+V4c53edEjtGLiEh0Se3Ri4hIRAp6EZGUS1zQm9laM9trZv1mdl/c9ZRjZj1mts3MdpvZLjP7/bC93cz+3cx+FH5fGLabmX063L/nzOzmvPf6QLj+j8zsA3ntt5jZD8PXfNrCZ94V28Yc7nvWzJ4xs2+F8yvM7Kmwzq+FN8nDzOrC+f5w+fK897g/bN9rZnfmtRf8OSi2jTna3zYz+7qZ7TGz583s9rQfZzP7w/DneqeZfdXM6tN2nM3sC2Z21Mx25rXFdlxLbaOo4DFzyfgiuKnaPmAlkAN2AKvjrqtMzV3AzeF0C/ACsBr4K+C+sP0+4BPh9N3AdwgeD7oGeCpsbwf2h98XhtMLw2U/CNe18LV3he0FtzGH+/4R4BHgW+H8Y8CGcPpzwG+F078NfC6c3gB8LZxeHR7jOmBFeOyzpX4Oim1jjvb3n4DfCKdzQFuajzPBA4deBBry/u0/mLbjDPw0cDOwM68ttuNabBsl92Gu/hNU6B/8dmBr3vz9wP1x1zXDffhXgufv7gW6wrYuYG84/RCwMW/9veHyjcBDee0PhW1dwJ689gvrFdvGHO1nN/Bd4OeAb4U/lK8BNdOPJcGdUW8Pp2vC9Wz68Z1ar9jPQaltzMH+LiAIPZvWntrjzOtPlmsPj9u3gDvTeJyB5bwx6GM7rsW2Uar+pA3dRHms4bwV/qn6FuAp4Ap3PxwuOgJcEU4X28dS7QMF2imxjbnwN8AfA5Ph/CLghAePmoQ31lnsUZQz/bcotY3ZtgIYBL5owXDV582siRQfZ3c/BPw18DJwmOC4PU26j/OUOI/rjHMwaUGfWGbWDHwD+AN3H85f5sGv5Vm9znUutjHFzN4FHHX3p+die/NEDcGf93/v7m8BzhD8uX1BCo/zQmA9wS+5pUATsHYutj2fJOG4Ji3oozzWcN4xs1qCkP+Ku/9L2PyqmXWFy7uAo2F7sX0s1d5doL3UNmbbW4F1ZnYAeJRg+OZvgTYLHjU5vc5ij6Kc6b/FsRLbmG0DwIC7PxXOf50g+NN8nN8OvOjug+4+BvwLwbFP83GeEudxnXEOJi3oozzWcF4Jz6D/I/C8u38qb1H+4xc/QDB2P9X+/vDM+hrgZPjn21bgnWa2MOxJvZNgXPIwMGxma8JtvX/aexXaxqxy9/vdvdvdlxMcoyfc/b3ANoJHTU6vp9ijKDcDG8KrNVYAqwhOXBX8OQhfU2wbs8rdjwAHzezNYdPbgN2k+DgTDNmsMbPGsKapfU7tcc4T53Etto3iZvMExiydFLmb4MqVfcDH4q4nQr0/SfAn13PAs+HX3QTjjN8FfgT8B9Aerm/ApnD/fgj05r3XrwP94dev5bX3AjvD13yG1z/xXHAbc7z/d/D6VTcrCf4D9wOPA3Vhe3043x8uX5n3+o+F+7WX8GqEUj8HxbYxR/t6E9AXHutvElxdkerjDHwc2BPW9WWCK2dSdZyBrxKcgxgj+MvtQ3Ee11LbKPalWyCIiKRc0oZuRERkhhT0IiIpp6AXEUk5Bb2ISMop6EVEUk5BLyKScgp6EZGU+3/cs9lKoZ8MMgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(1000000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABGUAAAE/CAYAAADvx35oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de7xkZX3n+8+XvnC/NHSHQRpojGhEo2A6qEdjjKCDJAFmwjgQk2AOk55cSHTkdQyOOV7ITZNJ1JwhKkeNJFEQ8UIfZQYRwVsUaQSRBtEWQRqBbpGLNpfuht/5Y61uqje7u4umdq3aVZ/361WvvS7Pque3dq1dz67fep6nUlVIkiRJkiRpuHbqOgBJkiRJkqRJZFJGkiRJkiSpAyZlJEmSJEmSOmBSRpIkSZIkqQMmZSRJkiRJkjpgUkaSJEmSJKkDJmUkSdLESnJLkmO6jkOSNJqSrEzy0q7j0PgyKTNkSZ6R5NokP0nyJ13HI21Pkrcm+deu45AkSZKGraqeVVVXdB2HxpdJmeF7A3B5Ve1ZVf/QdTBTJZmT5C+S/LBNHF2TZJ9pyl2WpJLM7dm2JMnlSR5I8u1+7jwmuSnJ06fZ/qok/9Y+1xXbOP532jj+S8+2JHlHkrvbxzuSpGf/EUmubp/76iRHDONYSZIkSZJ6mZQZvkOAlVvbmWTOEGOZztuA/wN4IbAX8NvAQ70FkrwamDfNsecB1wD7AW8CLkyyaGsVJflZYE5VfWea3T8G3gW8fRvHLwD+O4//fS4DTgSeCzwH+HXgv7bHzAcuAv4VWACcC1zUbp+xY5+o3mTXsLRJJd8TJE2kJDsneVd7U+KH7fLO7b6FST6d5N4kP07ypU3vl0n+NMnt7Y2Mm5Ic3e2ZSJIGadMw17b3+MeS/Gv7nv+tJE9P8sYka5LcluQVPccdmuSLbdnPJTnb3ueajh/AhijJ54FfAf5nkp+2f8QfSvKeJBcnWQf8SpJfbXuo3N/+cb+15zmWtD1Dfrfdd0+S30/yi0mua/9h/J9T6v0/k9zYlr0kySFbiW8B8Drg96rq1mpcX1UP9ZTZG3gLTY+f3mOfDjwPeEtVPVhVHwe+BfzGNn4lvwpcPN2OqvpcVV0A/HAbx/818A/Aj6ZsPxX4u6paXVW3A38HvKbd91JgLvCuqnq47a0U4GUzfOw2tW/yF7Zv8vcDr0myU5Izk3yv7XlzQZJ92/LnJjmjXT6wvSb+qF3/2fZDw05JFrQfJNa2r/+nkyzuqfeKJH+Z5CvAA8BT2wbkC20DcimwsJ9zaJ/vpUlWJzmjbZzuSPK7Pfv7ubZPTfKDJD9K8qZ+65akJ+lNwAuAI2iS60cBf9buOwNYDSwC9qe5IVBJngGcDvxiVe0J/HvgluGGLUkaol8H/oXmBu01wCU0n6kPBM4C3tdT9iPA12luWL+V5ma39DgmZYaoql4GfAk4var26Okh8pvAXwJ7Al8G1gG/A+xDk7j4gyQnTnm65wOHAf+ZpkfJm4BjgGcBr0ryywBJTqD55/E/0vwz+SWaHi3T+XlgI3BSkjuTfGfTB/0efwW8B7hzyvZnATdX1U96tn2z3b41xwGf2cb+rUpyFLAUeO80u5/V1j1dHM8Crquq6tl/3ZT9M3FsP04ALqR53T8M/DFNz5tfBp4C3AOc3Zb9Ak2SiHb/zcBLeta/VFWP0vyN/xNND62DgQeBLZJ2NA3EMprr71aaBuRqmmTMn9MkmzZrk3+/uY3z+HfA3jSN02nA2W3CD/q7tl8MPAM4Gnhzkmduoy5JGpRXA2dV1ZqqWkvTc3TTP9AbgAOAQ6pqQ1V9qW0LHgF2Bg5PMq+qbqmq73USvSRpGL5UVZdU1UbgYzSfr95eVRuA84ElSfZJcjDwi8Cbq2p9VX0ZWN5d2BplJmVGw0VV9ZWqerSqHqqqK6rqW+36dTRJlF+ecsyft2U/S/NB97z2H8nbaRIvR7blfh/466q6sX3z+CvgiK30lllM82H66cChwEnAW5O8HCDJUuBFwP8zzbF7APdN2XYfzQf9x0myG80b1RVb+6VsTZohXv9Ik9x6tI9Y7gP2SJI+4pypY/vx1ar6VPu6P0jz2r2p7XnzME2G/aQ0Q5u+ALw4Tff5lwB/Q/PaQHOtfAGgqu6uqo9X1QNtwuwvefy19KGqWtleHwfQvC7/d9sb6IvA/9dbuKqeU1Uf2cZ5bKD5YLOhqi4GfkqTZKHPa/ttbW+rb9Iktp7bzy9Pkp6kp9Akpje5td0G8LfAKuCzSW5OciZAVa2i6WH6VmBNkvOTPAVJ0ri6q2f5QeBHVfVIzzo0nwmeAvy4qh7oKX/bEOLTLGRSZjRs8Qea5PlpJsxdm+Q+mg/nU4eQTH1DmLq+R7t8CPDudljTvTRztYSmF8NUm95Izmo/FF9Hk/E9rv3w/4/Aa9sP71P9lGYOml57AT+Zpiw0vSD+rU02PFF/SNNj5Wtb2T81lr2An7Z3NbcX50wd24+pb9SHAJ/see1upLkru397J3YdTTf7XwI+Dfyw7Uq/OSmTZLck70tyazss6ovAPtly7qLeep8C3FNV63q29X5I6cfdU66RB2ivxz6v7TunO1aSZtgPad53Nzm43UZV/aSqzqiqpwLHA69PO3dMVX2kql7cHlvAO4YbtiRpBN0B7NveiN7koK6C0WgzKTMapn5o/whN97aDqmpvmiE6O/otPrcB/7Wq9ul57FpV/zZN2eumiWfT8l40w4U+muRO4Kp2++okv0Qz2e5Tk/T2jHkuW5/U+Di2Mp9MH44G/kM7xOpOmomJ/65nLp2VbNm7ojeOlcBzpvReec6U/TNxbD+mXge3Aa+c8trt0vaGgibxchIwv932BZqhRguAa9syZ9D0Unl+Ve3FY0Oces+ht947gAVJdu/ZdvATOIftGeS1LUmDdB7wZ0kWJVkIvJlmYneS/FqSp7Xv//fRJMgfTfKMJC9LMyHwQzQ3N6brwSlJmiBVdSuwgmbUwfwkL6SZj0Z6HJMyo2lPmu5uD7Vzp2xr/o7teS/wxiTPgmai3iT/abqCbe+LLwFvSvMtFM8ETqbphXEfTS+KI9rHce1hvwBc2c6Pcy3wliS7JPkPNAmLj28lrleyjflk0nw19y40E+vu1D7npm98eg3wzJ5YVtCM/d80Kew/09zFPLDtRn4G8KF23xU0/0z/SXuOp7fbPz/Dx+6I9wJ/uWmoWftB4YSe/V+gmWDyiz3xnQ58uacb5Z40HxLuTTNJ8Fu2VWFPA/K2tgF5MYNtQAZ5bUvSIP0FzfvfdTQT1X+j3QbNHG6fo+kR+VXgH6vqcpr5ZN5OM+H8ncDPAG8cbtiSpBH1appvtL2bpj35KLAjowQ05ob+tbvqyx/yWM+PLwAX0EyM+oRV1SeT7AGc3364vw+4lGZiqumcAnyA5s1jDc3cIpe1+zYPK2kTJgB39QxVOZkmCXEP8APgpHayxC0keTbNsJ4fbCP036aZoHaTB2m+gvo1VXXvlOdbD9xfVZvmc3kf8FSaf6oB3t9uo6rWtxPLvp/mH+kbgROrav1MHtvG+VOani9f2sZ593o3TS+Sz7ZJnjU0b+YXtfu/QJPk2JSU+TKwW886NJNAf4TmA8MPab4RaurEulP9Js3v+sc0Hz7+mZ7rL8lK4K+q6sN9nkevgV3bkjQIVbWkZ/VP2sfUMu8E3jnN9utovqVJkjSmetqJz03Z/jlgSc/6Rnp6gLc3vH9p03qSjwLfnsFQNUul/+kupMFI8gZgYVW9YbuFJUmSJGmWSfKLNDc5vw+8AvgU8MKquqbTwDRy7CmjLtzClG/0kSRJkqQx8u+ATwD7AauBPzAho+nYU0aSJEmSJKkDTvQrSZIkaeIkOTbJTUlWJTmz63gkTSZ7ykiSJEmaKEnmAN8BXk4ztOQq4JSquqHTwCRNnM7mlFm4cGEtWbKkq+olaWRdffXVP6qqRV3H0TXbCUmanu3EQBwFrKqqmwGSnA+cAGw1KTM/O9cu7N6s7LHrEEKEhxc8NrDh5xc87ktNp/XtBx/7Ysud7pgz8JikcdX79wb9/8314+rrHt7q+3ZnSZklS5awYsWKrqqXpJGV5NauYxgFthOSND3biYE4ELitZ3018PxtHbALu/P8HA3Ao0uPnLnIenzvpPmbl7/+G+/r65gXXfcfNy/v9td7DzwmaVz1/r1B/39z/ZhzwKqtvm/77UuSJEmSNI0ky4BlALuwW8fRSBpHTvQrSZIkadLcDhzUs7643baFqjqnqpZW1dJ57Dy04CRNDpMykiRJkibNVcBhSQ5NMh84GVjecUySJpDDlyRJkiRNlKramOR04BJgDvDBqlrZcViSJpBJGUmSJEkTp6ouBi7uOg5Jk83hS5IkSZIkSR0wKSNJkiRJktSB7SZlknwwyZok129lf5L8Q5JVSa5L8rzBhylJGlW2E5IkSdKO6aenzIeAY7ex/5XAYe1jGfCeJx+WJGkW+RC2E5IkSdITtt2kTFV9EfjxNoqcAPxzNb4G7JPkgEEFKEkabbYTkiRJ0o4ZxJwyBwK39ayvbrc9TpJlSVYkWbF27doBVC1JmgVsJyRJkqRpDHWi36o6p6qWVtXSRYsWDbNqSdIsYDshSRpZu+1CfuFZ5Bee1XUkksbIIJIytwMH9awvbrdJkgS2E5IkSdK0BpGUWQ78TvvtGi8A7quqOwbwvJKk8WA7IUmSJE1j7vYKJDkPeCmwMMlq4C3APICqei9wMXAcsAp4APjdmQpWkjR6bCckSRNhp/DIHvO7jkLSmNluUqaqTtnO/gL+aGARSZJmFdsJSZIkaccMdaJfSZIkSZIkNUzKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdWBu1wFIkiRJkjTTHth/5y3W7z9kcH0U5q57bHnhygcH9rwaf/aUkSRJkiRJ6oBJGUmSJEmSpA44fEmSJEmSNPYe3jNbrK875JGBPfe8e3v6O6wc2NNqAthTRpIkSdLYSvLBJGuSXN+zbd8klyb5bvtzQZcxSppcJmUkSZIkjbMPAcdO2XYmcFlVHQZc1q5L0tA5fEmSJEnS2KqqLyZZMmXzCcBL2+VzgSuAPx1aUOrErvc8usX63t+eM7DnnvNgDey5NFnsKSNJkiRp0uxfVXe0y3cC+3cZjKTJZVJGkiRJ0sSqqgKm7eaQZFmSFUlWrN+wbsiRSZoEJmUkSZIkTZq7khwA0P5cM12hqjqnqpZW1dL583YfaoCSJoNzykiSJEmaNMuBU4G3tz8v6jYcDcMud6+fst5RIFIPe8pIkiRJGltJzgO+Cjwjyeokp9EkY16e5LvAMe26JA2dPWUkSZIkja2qOmUru44eaiCSNA2TMpIkSZK0HY/O24kH9t8ZgIf3zIzVs9uPHpmx55Y0ehy+JEmSJEmS1AGTMpIkSZIkSR1w+JIkSZIkbccj8+H+Q5p72usOmcEhRtfM6Vl5dObqkTQS7CkjSZIkSZLUAZMykiRJkiRJHXD4kiRJkiRtRx6Bne8tAB7Zec52Su+4eQ84ZEmaJPaUkSRJkiRJ6oBJGUmSJEmSpA6YlJEkSZIkSeqAc8pIkiRJ0nbMfehRFnznIQAWfGc4dd7F/OFUJKkz9pSRJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6oBJGUmSJEmSpA6YlJEkSZIkSeqASRlJkiRJkqQOmJSRJEmSJEnqgEkZSZIkSZKkDvSVlElybJKbkqxKcuY0+w9OcnmSa5Jcl+S4wYcqSRpVthOSpHFXO4VHdp7DIzvPYcNuczc/dvS5eh+9z/fonJ02PySNv+3+pSeZA5wNvBI4HDglyeFTiv0ZcEFVHQmcDPzjoAOVJI0m2wlJkiRpx/STfj0KWFVVN1fVeuB84IQpZQrYq13eG/jh4EKUJI042wlJkiRpB/TT3+5A4Lae9dXA86eUeSvw2SR/DOwOHDOQ6CRJs4HthCRJkrQDdmwQ5OOdAnyoqv4uyQuBf0ny7Kp6tLdQkmXAMoCDDz54QFVLkmYB2wlJ0qy2cdew9rnzAVh3yCObt//sx574cz28YN4W67e/7LHlfa/p/Yi2RTMpaQz1M3zpduCgnvXF7bZepwEXAFTVV4FdgIVTn6iqzqmqpVW1dNGiRTsWsSRp1NhOSJJGUpKD2onmb0iyMslr2+37Jrk0yXfbnwu6jlXSZOonKXMVcFiSQ5PMp5mgcfmUMj8AjgZI8kyaf7bXDjJQSdLIsp2QJI2qjcAZVXU48ALgj9rJ6M8ELquqw4DL2nVJGrrtJmWqaiNwOnAJcCPNt2esTHJWkuPbYmcAv5fkm8B5wGuqqmYqaEnS6LCdkCSNqqq6o6q+0S7/hKadOpBmQvpz22LnAid2E6GkSdfXnDJVdTFw8ZRtb+5ZvgF40WBDkyTNFrYTkqRRl2QJcCRwJbB/Vd3R7roT2H8rx2ye62zeXo5wkjR4/QxfkiRJkqRZK8kewMeB11XV/b372p6b0/be7J3rbM6uuw8hUkmTZlDfviRJkiRJIyfJPJqEzIer6hPt5ruSHFBVdyQ5AFgzqPruedoum5d/fORj35504OcHVYOkcWJPGUmSJEljKUmADwA3VtXf9+xaDpzaLp8KXDTs2CQJ7CkjSZIkaXy9CPht4FtJrm23/Xfg7cAFSU4DbgVe1VF8kiacSRlJkiRJY6mqvgxkK7uPHmYskjQdhy9JkiRJkiR1wKSMJEmSJElSBxy+JEmSJEk76Pu/vvMW6zVn2m/X5o4XbW0UFWzlG7klTQB7ykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wDllJEmSJGkHPbpLf/PBPLqz88ZIejx7ykiSJEmSJHXApIwkSZIkSVIHHL4kSZIkSdsx96Fiv5UbANjnezN3b3vuug2bl398xJwZq0fSaLCnjCRJkiRJUgdMykiSJEmSJHXA4UuSJEmStB15pJi3biMA89YNq1aHL0njzp4ykiRJkiRJHTApI0mSJEmS1AGTMpIkSZIkSR0wKSNJkiRJktQBkzKSJEmSJEkdMCkjSZIkSZLUAZMykiRJkiRJHTApI0mSJEmS1AGTMpIkSZIkSR0wKSNJkiRJktQBkzKSJEmSxlKSXZJ8Pck3k6xM8rZ2+6FJrkyyKslHk8zvOlZJk8mkjCRJkqRx9TDwsqp6LnAEcGySFwDvAN5ZVU8D7gFO6zBGSRPMpIwkSZKksVSNn7ar89pHAS8DLmy3nwuc2EF4kmRSRpIkSdL4SjInybXAGuBS4HvAvVW1sS2yGjiwq/gkTTaTMpIkSZLGVlU9UlVHAIuBo4Cf6/fYJMuSrEiyYv2GdTMWo6TJZVJGkiRJ0tirqnuBy4EXAvskmdvuWgzcvpVjzqmqpVW1dP683YcUqaRJYlJGkiRJ0lhKsijJPu3yrsDLgRtpkjMntcVOBS7qJkJJk27u9otIkiRJ0qx0AHBukjk0N6QvqKpPJ7kBOD/JXwDXAB/oMkhJk8ukjCRJkqSxVFXXAUdOs/1mmvllJKlTDl+SJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6kBfSZkkxya5KcmqJGdupcyrktyQZGWSjww2TEnSKLOdkCRJkp647U70285UfjbN18etBq5KsryqbugpcxjwRuBFVXVPkp+ZqYAlSaPFdkKSJEnaMf30lDkKWFVVN1fVeuB84IQpZX4POLuq7gGoqjWDDVOSNMJsJyRJkqQd0E9S5kDgtp711e22Xk8Hnp7kK0m+luTYQQUoSRp5thOSJEnSDtju8KUn8DyHAS8FFgNfTPLzVXVvb6Eky4BlAAcffPCAqpYkzQK2E5IkSdIU/fSUuR04qGd9cbut12pgeVVtqKrvA9+h+ed7C1V1TlUtraqlixYt2tGYJUmjxXZCkiRJ2gH9JGWuAg5LcmiS+cDJwPIpZT5Fc/eTJAtpuqnfPMA4JUmjy3ZCkiRJ2gHbTcpU1UbgdOAS4EbggqpameSsJMe3xS4B7k5yA3A58H9V1d0zFbQkaXTYTkiSJEk7pq85ZarqYuDiKdve3LNcwOvbhyRpwthOSJIkSU9cP8OXJEmSJEmSNGAmZSRJkiRJkjpgUkaSJEmSJKkDJmUkSZIkSZI6YFJGkiRJkiSpAyZlJEmSJEmSOmBSRpIkSZIkqQMmZSRJkiRJkjpgUkaSJEmSJKkDJmUkSZIkSZI6YFJGkiRJkiSpAyZlJEmSJI21JHOSXJPk0+36oUmuTLIqyUeTzO86RkmTyaSMJEmSpHH3WuDGnvV3AO+sqqcB9wCndRKVpIlnUkaSJEnS2EqyGPhV4P3teoCXARe2Rc4FTuwmOkmTzqSMJEmSpHH2LuANwKPt+n7AvVW1sV1fDRzYRWCSZFJGkiRJ0lhK8mvAmqq6egePX5ZkRZIV6zesG3B0kgRzuw5AkiRJkmbIi4DjkxwH7ALsBbwb2CfJ3La3zGLg9ukOrqpzgHMA9trzwBpOyJImiT1lJEmSJI2lqnpjVS2uqiXAycDnq+rVwOXASW2xU4GLOgpR0oQzKSNJkiRp0vwp8Pokq2jmmPlAx/FImlAOX5IkSZI09qrqCuCKdvlm4Kgu45EksKeMJEmSJElSJ0zKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdcCkjCRJkiRJUgf6SsokOTbJTUlWJTlzG+V+I0klWTq4ECVJo852QpIkSXritpuUSTIHOBt4JXA4cEqSw6cptyfwWuDKQQcpSRpdthOSJEnSjumnp8xRwKqqurmq1gPnAydMU+7PgXcADw0wPknS6LOdkCRJknZAP0mZA4HbetZXt9s2S/I84KCq+swAY5MkzQ62E5KkkZXkliTfSnJtkhXttn2TXJrku+3PBV3HKWkyPemJfpPsBPw9cEYfZZclWZFkxdq1a59s1ZKkWcB2QpI0An6lqo6oqk1zmp0JXFZVhwGXteuSNHT9JGVuBw7qWV/cbttkT+DZwBVJbgFeACyfbhLHqjqnqpZW1dJFixbteNSSpFFiOyFJmm1OAM5tl88FTuwwFkkTrJ+kzFXAYUkOTTIfOBlYvmlnVd1XVQuraklVLQG+BhxfVStmJGJJ0qixnZAkjbICPpvk6iTL2m37V9Ud7fKdwP7THdjbg3P9hnXDiFXShJm7vQJVtTHJ6cAlwBzgg1W1MslZwIqqWr7tZ5AkjTPbCUnSiHtxVd2e5GeAS5N8u3dnVVWSmu7AqjoHOAdgrz0PnLaMJD0Z203KAFTVxcDFU7a9eStlX/rkw5IkzSa2E5KkUVVVt7c/1yT5JM23Bt6V5ICquiPJAcCaToOUNLGe9ES/kiRJkjSKkuyeZM9Ny8ArgOtphtme2hY7FbiomwglTbq+espIkiRJ0iy0P/DJJNB89vlIVf3vJFcBFyQ5DbgVeFWHMUqaYCZlJEmSJI2lqroZeO402+8Gjh5+RJK0JYcvSZIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXAOWUkSZIkSRPn+8fvvHn52v/8rs3Lz73wdZuXn/rJh4cakyaPPWUkSZIkSZI6YFJGkiRJkiSpAw5fkiRJkiRNnDySzcv//vrf3Lx8yMUbuwhHE8qeMpIkSZIkSR0wKSNJkiRJktQBhy9JkiRJkibOks889NjKZ/bu2fPI0GPR5LKnjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEkaW0n2SXJhkm8nuTHJC5Psm+TSJN9tfy7oOk5Jk8mkjCRJkqRx9m7gf1fVzwHPBW4EzgQuq6rDgMvadUkaOpMykiRJksZSkr2BlwAfAKiq9VV1L3ACcG5b7FzgxG4ilDTpTMpIkiRJGleHAmuBf0pyTZL3J9kd2L+q7mjL3Ans31mEkiaaSRlJkiRJ42ou8DzgPVV1JLCOKUOVqqqAmu7gJMuSrEiyYv2GdTMerKTJY1JGkiRJ0rhaDayuqivb9QtpkjR3JTkAoP25ZrqDq+qcqlpaVUvnz9t9KAFLmiwmZSRJkiSNpaq6E7gtyTPaTUcDNwDLgVPbbacCF3UQniQxt+sAJEmSJGkG/THw4STzgZuB36W5OX1BktOAW4FXdRifpAlmUkaSJEnS2Kqqa4Gl0+w6etixSNJUDl+SJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6oBzykiSJEnSdmzYfSfuWrorAD9d8siM1bPvtb33zR+dsXokjQZ7ykiSJEmSJHWgr6RMkmOT3JRkVZIzp9n/+iQ3JLkuyWVJDhl8qJKkUWU7IUmSJD1x203KJJkDnA28EjgcOCXJ4VOKXQMsrarnABcCfzPoQCVJo8l2QpIkSdox/fSUOQpYVVU3V9V64HzghN4CVXV5VT3Qrn4NWDzYMCVJI8x2QpIkSdoB/SRlDgRu61lf3W7bmtOA//VkgpIkzSq2E5IkSdIOGOi3LyX5LWAp8Mtb2b8MWAZw8MEHD7JqSdIsYDshSZqt5j5YLPzWegD2vSEzVs+chzduXv7xEX5ZrjTu+ukpcztwUM/64nbbFpIcA7wJOL6qHp7uiarqnKpaWlVLFy1atCPxSpJGj+2EJEmStAP6ScpcBRyW5NAk84GTgeW9BZIcCbyP5h/tNYMPU5I0wmwnJEmSpB2w3aRMVW0ETgcuAW4ELqiqlUnOSnJ8W+xvgT2AjyW5NsnyrTydJGnM2E5IkiRJO6avQYpVdTFw8ZRtb+5ZPmbAcUmSZhHbCUnSuMujxZyHHwFgTsexSBof/QxfkiRJkgW3KVQAAA29SURBVCRJ0oCZlJEkSZIkSeqASRlJkiRJkqQOmJSRJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6oBJGUmSJEmSpA6YlJEkSZIkSeqASRlJkiRJkqQOmJSRJEmSNJaSPCPJtT2P+5O8Lsm+SS5N8t3254KuY5U0mUzKSJIkSRpLVXVTVR1RVUcAvwA8AHwSOBO4rKoOAy5r1yVp6EzKSJIkSZoERwPfq6pbgROAc9vt5wIndhaVpIlmUkaSJEnSJDgZOK9d3r+q7miX7wT27yYkSZPOpIwkSZKksZZkPnA88LGp+6qqgNrKccuSrEiyYv2GdTMcpaRJZFJGkiRJ0rh7JfCNqrqrXb8ryQEA7c810x1UVedU1dKqWjp/3u5DClXSJDEpI0mSJGncncJjQ5cAlgOntsunAhcNPSJJwqSMJEmSpDGWZHfg5cAneja/HXh5ku8Cx7TrkjR0c7sOQJIkSZJmSlWtA/absu1umm9jkqRO2VNGkiRJkiSpAyZlJEmSJEmSOmBSRpIkSZIkqQPOKSNJkiRJkp6wB/bfefPyTw94rM/HThsHV8fC6x8c3JONIHvKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXAOWUkSZIkSdIT9sCix/p53P+sDZuX8/CT7P+RxxYXXv/knmrU2VNGkiRJkiSpAyZlJEmSJEmSOuDwJUmSJEmS9ITtcs+jm5d3+/68zctz1ncRzexkTxlJkiRJkqQOmJSRJEmSJEnqgMOXJEmSJEnSE7bH7Q/3LHcYyCxmTxlJkiRJkqQOmJSRJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6oBJGUmSJEmSpA6YlJEkSZIkSeqASRlJkiRJkqQOmJSRJEmSJEnqgEkZSZIkSZKkDvSVlElybJKbkqxKcuY0+3dO8tF2/5VJlgw6UEnS6LKdkCRJkp647SZlkswBzgZeCRwOnJLk8CnFTgPuqaqnAe8E3jHoQCVJo8l2QpI0ypL8tyQrk1yf5LwkuyQ5tL1JsKq9aTC/6zglTaZ+esocBayqqpuraj1wPnDClDInAOe2yxcCRyfJ4MKUJI0w2wlJ0khKciDwJ8DSqno2MAc4mebmwDvbmwX30Nw8kKSh6ycpcyBwW8/66nbbtGWqaiNwH7Df1CdKsizJiiQr1q5du2MRS5JGje2EJGmUzQV2TTIX2A24A3gZzU0CaG4anNhRbJIm3FAn+q2qc6pqaVUtXbRo0TCrliTNArYTkqRBqqrbgf8B/IAmGXMfcDVwb3uTAKa/mQBsebNg/YZ1wwhZ0oTpJylzO3BQz/ridtu0ZdoM9N7A3YMIUJI08mwnJEkjKckCmiG0hwJPAXYHju33+N6bBfPn7T5DUUqaZP0kZa4CDmsnw5pPMwZz+ZQyy4FT2+WTgM9XVQ0uTEnSCLOdkCSNqmOA71fV2qraAHwCeBGwT3uTAKa/mSBJQ7HdpEzbre904BLgRuCCqlqZ5Kwkx7fFPgDsl2QV8HrgcV+HKkkaT7YTkqQR9gPgBUl2ayeYPxq4Abic5iYBNDcNLuooPkkTbu72i0BVXQxcPGXbm3uWHwL+02BDkyTNFrYTkqRRVFVXJrkQ+AawEbgGOAf4DHB+kr9ot32guyglTbK+kjKSJEmSNBtV1VuAt0zZfDNwVAfhSNIWhvrtS5IkSZIkSWqYlJEkSZIkSeqASRlJkiRJkqQOmJSRJEmSJEnqgEkZSZIkSZKkDpiUkSRJkiRJ6kCqqpuKk7XArTt4+ELgRwMMZ9RN0vlO0rmC5zvOnsy5HlJViwYZzGxkO/GETNL5TtK5guc7zmwnZpmedqnr63TS6x+FGCa9/lGIoev6n2gMW33f7iwp82QkWVFVS7uOY1gm6Xwn6VzB8x1nk3Suo2jSfv+TdL6TdK7g+Y6zSTrXcdP1azfp9Y9CDJNe/yjE0HX9g4zB4UuSJEmSJEkdMCkjSZIkSZLUgdmalDmn6wCGbJLOd5LOFTzfcTZJ5zqKJu33P0nnO0nnCp7vOJukcx03Xb92k14/dB/DpNcP3cfQdf0woBhm5ZwykiRJkiRJs91s7SkjSZIkSZI0q410UibJsUluSrIqyZnT7N85yUfb/VcmWTL8KAejj3N9fZIbklyX5LIkh3QR56Bs73x7yv1Gkkoyq7+doJ/zTfKq9jVemeQjw45xUPq4lg9OcnmSa9rr+bgu4hyUJB9MsibJ9VvZnyT/0P4+rkvyvGHHOM5sJ7bYbzsxi9lObLF/bNoJ24jx0u/70oDrfNw1lGTfJJcm+W77c8EM1n9Q+/e46b3ntcOMIckuSb6e5Jtt/W9rtx/atuur2nZ+/kzU3xPHnPY96dMd1X9Lkm8luTbJinbbMK+DfZJcmOTbSW5M8sIh1/+M9tw3Pe5P8rohx/Df2mvw+iTntdfmYK6DqhrJBzAH+B7wVGA+8E3g8Cll/hB4b7t8MvDRruOewXP9FWC3dvkPZuu59nu+bbk9gS8CXwOWdh33DL++hwHXAAva9Z/pOu4ZPNdzgD9olw8Hbuk67id5zi8Bngdcv5X9xwH/CwjwAuDKrmMel4fthO2E7UT3sc/QuY5NO2EbMT6Pft+XZqDex11DwN8AZ7bLZwLvmMH6DwCe1y7vCXyn/bscSgzt38Ye7fI84Mr2b+UC4OR2+3s3vWfM4O/h9cBHgE+368Ou/xZg4ZRtw7wOzgX+S7s8H9hnmPVPiWUOcCdwyBCvwwOB7wO79rz+rxnUdTDKPWWOAlZV1c1VtR44HzhhSpkTaC4QgAuBo5NkiDEOynbPtaour6oH2tWvAYuHHOMg9fPaAvw58A7goWEGNwP6Od/fA86uqnsAqmrNkGMclH7OtYC92uW9gR8OMb6Bq6ovAj/eRpETgH+uxteAfZIcMJzoxp7tRA/biVnNdmJLY9NO2EaMlX7flwZqK9dQb9t2LnDiDNZ/R1V9o13+CXAjzQfUocTQ/m38tF2d1z4KeBlNuz6j9QMkWQz8KvD+dj3DrH8bhvIaJNmbJjn4AYCqWl9V9w6r/mkcDXyvqm4dcgxzgV2TzAV2A+5gQNfBKCdlDgRu61lf3W6btkxVbQTuA/YbSnSD1c+59jqN5q7KbLXd82277x5UVZ8ZZmAzpJ/X9+nA05N8JcnXkhw7tOgGq59zfSvwW0lWAxcDfzyc0DrzRP++1T/bia2znZhdbCe29FYmp52wjZg9Rum12r+q7miX7wT2H0alaYYAH0nTW2VoMbRDh64F1gCX0vRYurdt12HmX4t3AW8AHm3X9xty/dAkoj6b5Ooky9ptw3oNDgXWAv/UDuF6f5Ldh1j/VCcD57XLQ4mhqm4H/gfwA5pkzH3A1QzoOhjlpIymkeS3gKXA33Ydy0xJshPw98AZXccyRHNpuqa/FDgF+H+T7NNpRDPnFOBDVbWYptv2v7SvuaQBsJ0YW7YTkh6nmnETM/51ukn2AD4OvK6q7h9mDFX1SFUdQdMD9Cjg52aqrqmS/BqwpqquHladW/Hiqnoe8Ergj5K8pHfnDL8Gc2mG0L2nqo4E1tEMFRpW/Zu1c7YcD3xs6r6ZjKGdq+YEmgTVU4DdgYHdHBnlBu524KCe9cXttmnLtN2I9gbuHkp0g9XPuZLkGOBNwPFV9fCQYpsJ2zvfPYFnA1ckuYVm3OjyzN5JHPt5fVcDy6tqQ1V9n2a87mFDim+Q+jnX02jGX1JVXwV2ARYOJbpu9PX3rR1iOzGF7YTtxCxgO7El24jZY5Req7s2DXNrf87ocMYk82gSMh+uqk90EQNAO2TmcuCFNEP95ra7ZvK1eBFwfNvWnE8zXOXdQ6wf2NxTY9PQ1U/SJKeG9RqsBlZX1ZXt+oU0SZqhXwM0SalvVNVd7fqwYjgG+H5Vra2qDcAnaK6NgVwHo5yUuQo4rJ3ReD5NN6XlU8osB05tl08CPt9myGab7Z5rkiOB99H8oz1bx5Fvss3zrar7qmphVS2pqiU0cyMcX1Urugn3SevnWv4Uzd1Pkiyk6aZ+8zCDHJB+zvUHNGNBSfJMmn+21w41yuFaDvxOGi8A7uvpZqknx3aih+2E7cQsYTuxJduI2aOfa3dYetu2U4GLZqqidv6UDwA3VtXfDzuGJIs29QpMsivwcpp5bS6naddntP6qemNVLW7bmpNp/o949bDqB0iye5I9Ny0DrwCuZ0ivQVXdCdyW5BntpqOBG4ZV/xSn8NjQJYYYww+AFyTZrf2b2PQ7GMx1UEOYIXlHHzRdVr9DM27wTe22s2j+8YKmkf4YsAr4OvDUrmOewXP9HHAXcG37WN51zDN5vlPKXsEs/laNPl/f0HTFvwH4Fu0s3rPx0ce5Hg58heZbC64FXtF1zE/yfM+jGVu6geZOwmnA7wO/3/Pant3+Pr4126/lUXvYTthOtNttJ2bRY5LaCduI8XpMd+0Ooc7prqH9gMuA77bv/fvOYP0vphkScl1P+3LcsGIAnkPzzXPX0SQi3txuf2rbrq9q2/mdh/BavJTHvn1paPW3dX2zfazsed8c5nVwBLCifR0+BSwYZv1tDLvT9Hbeu2fbMH8HbwO+3V6H/wLsPKjrIG0FkiRJkiRJGqJRHr4kSZIkSZI0tkzKSJIkSZIkdcCkjCRJkiRJUgdMykiSJEmSJHXApIwkSZIkSVIHTMpIkiRJkiR1wKSMJEmSJElSB0zKSJIkSZIkdeD/BxOEwWE36/9eAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x360 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 1400000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = model.act(state, epsilon)\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > replay_initial:\n",
    "        loss = compute_td_loss(batch_size)\n",
    "        losses.append(loss.data.item())\n",
    "        \n",
    "    if frame_idx % 2 == 0:\n",
    "#         plot(frame_idx, all_rewards, losses)\n",
    "        img = state[0]\n",
    "        plot_game(frame_idx, num_frames, all_rewards, losses, img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
